import json
import pandas as pd
from multiprocessing import Pool
import pymongo
import time
import requests
from get_field_name.config import *
from get_field_name.save_to_db import *
# 连接数据库
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
table = db['fourth_name']

def get_fourth_field_name_list():
    data = pd.DataFrame(list(table.find()))
    data = data[["id", "n", "first_field_name", "second_field_name", "third_field_name"]]
    fourth_field_name_list = []
    for i in data.index:
        fourth_field_name_list.append({"id": data.loc[i][0], "fourth_name": data.loc[i][1], "first_name": data.loc[i][2], "second_name": data.loc[i][3], "third_name": data.loc[i][4]})
    # for n in third_field_name_list:
    #     print(n)
    return fourth_field_name_list
fourth_field_name_list = get_fourth_field_name_list()
failed_index = []

def get_fifth_field_name(url, index):
    try:
        response = requests.get(url, verify=False)
        if response.status_code == 200:  # 返回成功
            return response.content
        else:
            failed_index.append(index)
            return None
    except:
        time.sleep(10)


def main(i):
    url = "https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath=" + str(
        fourth_field_name_list[i]['id'])
    print("index:", i)
    print(fourth_field_name_list[i]['fourth_name'])
    Data = get_fifth_field_name(url, i)
    if Data is not None:
        tansformData = json.loads(Data)
        fifth_field_name = tansformData['ct']
        parent_field_name = tansformData['bc']
        # for pp in parent_field_name:
        #     print(pp)
        for ss in fifth_field_name:
            ss['first_field_name'] = fourth_field_name_list[i]['first_name']
            ss['second_field_name'] = fourth_field_name_list[i]['second_name']
            ss['third_field_name'] = fourth_field_name_list[i]['third_name']
            ss['fourth_field_name'] = parent_field_name[0]
        try:
            save_data("fifth_name_bk", fifth_field_name)
        except Exception:
            print("index:", i)



if __name__ == '__main__':
    pool = Pool(4)
    pool.map(main, [i for i in range(0, len(fourth_field_name_list))])
    pool.close()
    pool.join()
    print(failed_index)